package com.shujia.spark.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo3Map {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local")
    conf.setAppName("map")
    val sc = new SparkContext(conf)

    //读取文件
    val linesRDD: RDD[String] = sc.textFile("data/students.txt")

    /**
     * map算子：数据量不变，原RDD的行数和结果RDD的行数不变
     */

    val clazzRD: RDD[(String, Int)] = linesRDD.map(line => {
      val split: Array[String] = line.split(",")
      val clazz: String = split.last
      (clazz, 1)
    })

    //统计班级的人数
    val clazzNumRDD: RDD[(String, Int)] = clazzRD.reduceByKey(_ + _)

    clazzNumRDD.foreach(println)
  }
}
